var cheerio = require('cheerio'); //可以像jquer一样操作界面
var charset = require('superagent-charset'); //解决乱码问题:
var superagent = require('superagent'); //发起请求
var Async = require('async'); //异步抓取
charset(superagent);

var baseUrl = 'http://www.dytt8.net'; //迅雷首页链接
var newMovieLinkArr = []; //存放新电影的url
var errLength = []; //统计出错的链接数
var highScoreMovieArr = [] //高评分电影




const listPage = (url) => {
    return new Promise(function(resolve, reject){
        superagent
        .get(url)
        .charset('gb2312')
        .end(function (err, document) {
            // 常规的错误处理
            if (err) {
                reject(err)
                console.log('抓取' + page + '这条信息的时候出错了')
            }
            var $ = cheerio.load(document.text);
            resolve($)
        });
    })
}




const getAllMovieLink = function ($, host) {
    return new Promise(function(resolve, reject){
        try {
            let movieLinkArr = []
            const linkElem = $('.co_content2 ul a')
            for (var i = 1; i < 100; i++) {// linkElem.length; i++) {
                var url = host + linkElem.eq(i).attr('href')
                // 注意去重
                if (movieLinkArr.indexOf(url) == -1 && url !== 'undefined') {
                    movieLinkArr.push(url);
                }
            }
            resolve(movieLinkArr)
        } catch (error) {
            reject(error)
        }
    })
}



// 获取下载链接
const getDownloadLink = ($, cb) => {
    let downLink = $('#Zoom table a').text()
    const thunderUrl = $('#Zoom a').eq(0).attr('href')
    var movieName = $('.title_all h1 font').text()
    const desc = $('#Zoom span').children().first().html()
    const timeDom = $('.co_content8 ul').text()
    let time = ''
    const pic = $('#Zoom span').find('img')
    reg = /发布时间：((\w|-|\s)+)/ig;
    desc.replace(reg, function(s,value) {
        console.log(value, '=======');
        time = value
	})

    
    if (!downLink) downLink = '该电影暂无链接';
    var obj = {
        'downLink': downLink,
        'movieName': movieName,
        'desc': desc,
        'time': time,
        'pic': pic.eq(0) ? pic.eq(0).attr('src') : '',
        'thunder_url': thunderUrl
    }
    cb(obj)
}



const multithreading = (arr) => {
    return new Promise(function(resolve, reject){
        let array = []
        Async.mapLimit(arr, 5, function (url, callback) {
            var fetchStart = new Date().getTime();
            console.log('正在抓取的是', url)
            superagent
                .get(url)
                .charset('gb2312') //解决编码问题
                .end(function (err, ssres) {
                    if (err) {
                        callback(null);
                        return
                    }
                    var time = new Date().getTime() - fetchStart;
                    console.log('抓取 ' + url + ' 成功', '，耗时' + time + '毫秒');
                    var $ = cheerio.load(ssres.text);
                    getDownloadLink($, function(rs){
                        callback(null, rs);
                        array.push(rs)
                    })
                });
        }, function (err, result) {
            if(err) {
                console.log(err)
                reject(err)
            }
            resolve(result)
            // 爬虫结束后的回调，可以做一些统计结果
            console.log('抓包结束，一共抓取了-->' + arr.length + '条数据');
            console.log('出错-->' + errLength.length + '条数据');
            console.log('高评分电影：==》' + highScoreMovieArr.length + '部');
            return false;
        });
        
    })
}
const crawler = async () => {
    let $ = await listPage(baseUrl)
    // 170条电影链接，注意去重
    let arr = await getAllMovieLink($, baseUrl)
    let moreUrl = await multithreading(arr)
    // console.log(moreUrl);
    return moreUrl
    
}

module.exports = crawler